import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.swing.text.AttributeSet;
import javax.swing.text.Element;
import javax.swing.text.ElementIterator;
import javax.swing.text.html.HTML;
import javax.swing.text.html.HTMLDocument;
import javax.swing.text.html.HTMLEditorKit;



public class Engine implements Runnable {
	boolean stop = false;
	boolean end = false;
	BotConfig _cfg;
	String urlSearchPage;
	IProvider _prov;
	public Engine(BotConfig cfg, IProvider prov){
		_cfg = cfg;
		_prov = prov;
	}
	
	public void run() {
		try {
			System.out.println(_prov.getEnginName() + ": Go..");
			
			String absPath = _cfg.getOutPutFolder();
			File dir = new File(absPath);
			if(dir.exists() == false)
				dir.mkdirs();
			
			if(absPath.endsWith(File.separator) == false)
				absPath += File.separator;
			absPath = absPath + _prov.getEnginName();
			
			dir = new File(absPath);
			if(dir.exists() == false)
				dir.mkdirs();
			if(absPath.endsWith(File.separator) == false)
				absPath += File.separator;
			
			for (String useCase : _cfg.getUseCases()) {
				if(_cfg.isExtLoging()){
					System.out.println(_prov.getEnginName() + ": " + useCase + "...");
				}
				if(stop)
			    	break;				
				String fileUseCase = useCase.length() == 0 ? "general" : useCase;
				File dataFile = new File(absPath + fileUseCase.replace('+', '_').replace('|', '_') + ".txt");
				if(dataFile.exists())
					dataFile.delete();
				
				Hashtable<String, ArrayList<Integer>> ht = new Hashtable<String, ArrayList<Integer>>();					
				for (String categ : _cfg.getCategories()) {
					if(_cfg.isExtLoging()){
						System.out.println(_prov.getEnginName() + ":    " + categ);
					}
					if(stop)
				    	break;
					ht.put(categ, doCatCase(useCase, categ, absPath));
				}
				Saver saver = new Saver(dataFile, ht,_cfg.getFromYear(), _cfg.getToYear());
				new Thread(saver).start();
			}
			
			System.out.println(_prov.getEnginName() + ": Koniec!!!");
		} catch (Exception ex) {
			System.err.println(_prov.getEnginName() + ": " + ex.getMessage());
			for (StackTraceElement el : ex.getStackTrace()) {
				System.err.println(el.toString());
			}
		}
		end = true;
	}
	
	private String cookieVal = "";
	private ArrayList<Integer> doCatCase(String useCase, String categ, String dirAbsPath) throws Exception{
		InputStream is = null;
		ArrayList<Integer> resArr = new ArrayList<Integer>();
				
		int res = 0;
		String nextUrl = null;
		
		String saverDir = useCase + "_" + categ;
		saverDir = dirAbsPath + saverDir + File.separator;
		
		for(int y=_cfg.getFromYear();y<=_cfg.getToYear();++y){
			res = 0;
			nextUrl = null;
			URLConnection conn = null;	
			cookieVal = null;
			
			System.out.println(_prov.getEnginName() + ": rok - " + Integer.toString(y));
			
			do{
				try
				{
					if(stop)
				    	break;
					if(nextUrl == null) 
						conn = createConn(_prov.getSearchURL(y, useCase, categ));
					else
					{
						for(int i=0;i<30;++i){
							if(stop)
						    	break;
							Thread.sleep(1000);
						}
						if(stop)
					    	break;
						conn = createConn(nextUrl);
					}
				    
				    is = conn.getInputStream();
			    	cookieVal = retrieveCookie(conn, cookieVal);
						
				    
				    HTMLEditorKit htmlKit = new HTMLEditorKit();
				    HTMLDocument htmlDoc = (HTMLDocument) htmlKit.createDefaultDocument();
				    htmlDoc.putProperty("IgnoreCharsetDirective", new Boolean(true));
				    htmlKit.read(is, htmlDoc, 0);
				    
				    if(_cfg.isExtLoging()){
				    	FileOutputStream tSt = new FileOutputStream(_prov.getEnginName() + ".html");		    	
				    	htmlKit.write(tSt, htmlDoc, 0, htmlDoc.getLength());
					    tSt.flush();
					    tSt.close();
					}
				    if(nextUrl == null) {//pierwsza  strona
					    List<String> arInt = checkPattternInDoc(htmlDoc, _prov.getPositionCountSearch());
					    if(arInt != null){
					    	String strInt = arInt.get(0);
					    	StringBuffer sb = new StringBuffer();
				    		for(int i=0;i<strInt.length();++i)
				    			if(strInt.charAt(i) >= '0' && strInt.charAt(i) <= '9')
				    				sb.append(strInt.charAt(i));
				    		res = Integer.parseInt(sb.toString());
					    }			    
					    if(res == 0)
							res = _prov.finalPositionCountTest(htmlDoc);
				    }
				    
				    List<String> arDetItms = checkPattternInDoc(htmlDoc, _prov.getPositionDetailSearch());
				    if(arDetItms != null){
				    	for (String arDetItm : arDetItms) {
				    		if(stop)
						    	break;
				    		String[] infos = new String[_prov.getKeyWordCount()];
				    		for(int idx=0;idx<infos.length;++idx)
				    			infos[idx] = null;
				    		fillDetailInfo(_prov.getURL(arDetItm), infos);
				    		
				    		Saver.SaveDetailsInfo(saverDir, infos, _prov);
						}
				    }
				    
				    nextUrl = null;
				    List<String> arNxt = checkPattternInDoc(htmlDoc, _prov.getNextSearch());
				    if(arNxt != null){
				    	nextUrl = _prov.getURL(arNxt.get(0));
				    }
				}
				catch (Exception e) {
					if(e.getMessage().contains("503"))
					{
						//Thread.sleep(4000);
						//--y;
						System.out.println(_prov.getEnginName() + ": Spróbuj ponownie za jakiś czas...");
						System.err.println(_prov.getEnginName() + ": Spróbuj ponownie za jakiś czas...");
						throw e;
					}
					else if(e.getMessage().contains("Connection refused")){
						Thread.sleep(4000);
						--y;
					}
					else
						throw e;
				}
				finally
				{
					if(is != null)
						is.close();
					is = null;
				}	
			}
			while(nextUrl != null);
			resArr.add(new Integer(res));
			
			for(int i=0;i<30;++i){
				if(stop)
			    	break;
				Thread.sleep(1000);
			}
			if(stop)
		    	break;
		}
		return resArr;
	}
	
	protected void fillDetailInfo(String url, String[] infos) throws Exception
	{
		InputStream is = null;
		URLConnection conn = null;
		try
		{
			if(stop)
		    	return;
			conn = createConn(url);		
			is = conn.getInputStream();
			cookieVal = retrieveCookie(conn, cookieVal);
			
			HTMLEditorKit htmlKit = new HTMLEditorKit();
		    HTMLDocument htmlDoc = (HTMLDocument) htmlKit.createDefaultDocument();
		    htmlDoc.putProperty("IgnoreCharsetDirective", new Boolean(true));
		    htmlKit.read(is, htmlDoc, 0);
		    
		    boolean searchNext = false;
			for(int idx=0;idx < infos.length;++idx) {
				if(stop)
			    	return;
				if(infos[idx] != null)
					continue;
				List<String> arDtl = checkPattternInDoc(htmlDoc, _prov.getKeySearchDetails(idx));
				if(arDtl != null)
					infos[idx] = arDtl.get(0);
				else
					searchNext = true;
			}
			if(searchNext){
				if(stop)
			    	return;
			    List<String> arNxt = checkPattternInDoc(htmlDoc, _prov.getNextDetail());
			    if(arNxt != null){
			    	fillDetailInfo(_prov.getURL(arNxt.get(0)), infos);
			    }
			}
		}
		catch (Exception e) {
			if(e.getMessage().contains("503"))
			{
				System.out.println(_prov.getEnginName() + ": Spróbuj ponownie za jakiś czas...");
				System.err.println(_prov.getEnginName() + ": Spróbuj ponownie za jakiś czas...");
				throw e;
			}
			else if(e.getMessage().contains("Connection refused")){
				Thread.sleep(4000);
				fillDetailInfo(url, infos);
			}
			else
				throw e;
		}
		finally
		{
			if(is != null)
				is.close();
			is = null;
		}
	}
		
	public void stop(){
		stop = true;
	}
	public boolean isEnd() {
		return end;
	}
	
	
	private String retrieveCookie(URLConnection conn, String cookieValOld)
	{
		String cookieValue = cookieValOld;

		Map<String, List<String>> headers = conn.getHeaderFields();
		if(headers == null)
			return cookieValOld;
		List<String> values = headers.get("Set-Cookie");
		if(values == null)
			return cookieValOld;
		Iterator<String> iter = values.iterator();
		if(iter.hasNext() == false)
			return cookieValOld;
		while (iter.hasNext()) {
		     String v = iter.next();
		     if (cookieValue == null)
		         cookieValue = v;
		     else
		     {
		    	String[] cooks = v.split(";");
		    	for (String cook : cooks) {
		    		String[] tabs = cook.split("=");
			    	 if(tabs == null || tabs.length != 2 || 
			    			 tabs[0] == null || cookieValue.contains(tabs[0] + "=") == false)
			    		 cookieValue = cookieValue + ";" + v;
			    	 else {
			    		 Pattern cookiePat = Pattern.compile("^(.*" + tabs[0] + "=)([^=;]*)(.*)$");
			    		 Matcher m = cookiePat.matcher(cookieValue);
			    		 m.find();
			    		 cookieValue = m.group(1) + tabs[1] + m.group(3);
			    	 }
				}
		     }
		} 
		return cookieValue;
	}
	
	private URLConnection createConn(String urlStr) throws Exception
	{
		URL url = new URL(urlStr);
		URLConnection conn = url.openConnection();
	    if(cookieVal != null && cookieVal.length() > 0){
	    	conn.setRequestProperty("Cookie", cookieVal); 
	    }
	    conn.setRequestProperty("User-Agent","Mozilla/5.0 (Windows; U; Windows NT 5.1; pl; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7");
	    return conn;
	}
	
	private List<String> checkPattternInDoc(HTMLDocument htmlDoc, IProvider.SearchDetails searchDetails) throws Exception
	{
		if(searchDetails == null)
			return null;
		HTML.Tag ATag = searchDetails.ATagSearch;
		HTML.Tag BTag = searchDetails.BTagSearch;
		
		Pattern patSearch = searchDetails.get_searchpat();
		int grNum = searchDetails.RegExGroupNum;
		
		Pattern patSearchAt = searchDetails.get_searchpatAt();
		int grNumAt = searchDetails.RegExAttributeGroupNum;
		HTML.Attribute tTagAtribute = searchDetails.TTagAtribute;		
		
		List<String> ret = null;
		if(ATag != null){
	    	for (HTMLDocument.Iterator iterator = htmlDoc.getIterator(ATag); iterator != null && iterator.isValid(); iterator.next()) {		    	
		    	if(stop)
			    	break;
		    	boolean add = false;
		    	String addContent = "";
		    	
		    	if(patSearch != null) {
		    		
		    		int stIdx1 = iterator.getStartOffset();
		    		
			    	int edIdx1 = iterator.getEndOffset();
		    		String cont = htmlDoc.getText(stIdx1, edIdx1 -stIdx1);
		    		
			    	Matcher m = patSearch.matcher(cont);
			    	if(m.find()){
			    		if(grNum > 0){
			    			addContent = m.group(grNum);
			    			add = true;
			    		}
			    	}
			    	else
			    		continue;
		    	}
		    	if(tTagAtribute != null && patSearchAt != null) {
		    		AttributeSet attributes = iterator.getAttributes();
		    		String cont = (String) attributes.getAttribute(tTagAtribute);
		    		
		    		Matcher m = null;
			    	if(cont != null)
			    		m = patSearchAt.matcher(cont);
			    	if(m != null && m.find()){
			    		if(grNumAt > 0){
			    			addContent = m.group(grNumAt);
			    			add = true;
			    		}
			    	}
			    	else
			    		continue;
		    	}
		    	
		    	if(add){
		    		if(ret == null)
		    			ret = new ArrayList<String>();
		    		if(ret.contains(addContent) == false)
		    			ret.add(addContent);
		    	}
		    }
	    }
	    if(BTag != null){
	    	ElementIterator ei = new ElementIterator(htmlDoc);
	        Element e;
	        while ((e = ei.current()) != null) {
	        	if(stop)
	        		break;
	        	boolean add = false;
		    	String addContent = "";
		    	
		    	if(BTag.toString().compareToIgnoreCase(e.getName()) == 0){
		    		if(patSearch != null) {
		        		int stIdx = e.getStartOffset();
		        		int edIdx = e.getEndOffset();
		        		String cont = htmlDoc.getText(stIdx, edIdx -stIdx);
		        		Matcher m = patSearch.matcher(cont);
				    	if(m.find()){
				    		if(grNum > 0) {
				    			addContent = m.group(grNum);
				    			add = true;
				    		}
				    	}
				    	else {
				    		ei.next();
				    		continue;
				    	}
		    		}
		    		
		    		if(tTagAtribute != null && patSearchAt != null) {
			    		AttributeSet attributes = e.getAttributes();
			    		String cont = (String) attributes.getAttribute(tTagAtribute);
			    		
				    	Matcher m = null;
				    	if(cont != null)
				    		m = patSearchAt.matcher(cont);
				    	if(m != null && m.find()){
				    		if(grNumAt > 0) {				    			
				    			addContent = m.group(grNumAt);
				    			add = true;
				    		}
				    	}
				    	else {
				    		ei.next();
				    		continue;
				    	}
			    	}
			    }
	        	
	        	if(add){
		    		if(ret == null)
		    			ret = new ArrayList<String>();
		    		if(ret.contains(addContent) == false)
		    			ret.add(addContent);
		    	}
	        	ei.next();
	        }	
	    }
	    return ret;
	}
}
